<!DOCTYPE html> <html lang=en><!--
 Page saved with SingleFile 
 url: https://learnk8s.io/kubernetes-long-lived-connections 
 saved date: Sat Feb 29 2020 14:24:05 GMT+0800 (China Standard Time)
--><meta charset=utf-8><title>Load balancing and scaling long-lived connections in Kubernetes</title><meta name=description content="Kubernetes doesn't load balance long-lived connections and some Pods might receive more requests than others. Learn how to fix that."><meta name=author content=Learnk8s><meta name=viewport content="width=device-width, initial-scale=1.0"><meta http-equiv=X-UA-Compatible content="ie=edge"><meta name=twitter:card content=summary><meta name=twitter:site content=@learnk8s><meta property=fb:app_id content=398212777530104><link rel=alternate type=application/rss+xml title="Subscribe to Learnk8s RSS" href=https://learnk8s.io/rss.xml><meta property=og:site_name content=Learnk8s><meta name=pocket-site-verification content=1476398dfb5a771a94da9466e0bb43><meta property=og:url content=https://learnk8s.io/kubernetes-long-lived-connections><meta property=og:type content=website><meta property=og:title content="Load balancing and scaling long-lived connections in Kubernetes"><meta property=og:image content=https://learnk8s.io/a/4f9741344d5b7e302652468bc6be6b59.png><meta property=og:description content="Kubernetes doesn't load balance long-lived connections, and some Pods might receive more requests than others. If you're using HTTP/2, gRPC, RSockets, AMQP or any other long-lived connection such as a database connection, you might want to consider client-side load balancing."><link rel=canonical href=https://learnk8s.io/kubernetes-long-lived-connections><script type=application/ld+json>{"@context":"https://schema.org","@type":"BlogPosting","headline":"Load balancing and scaling long-lived connections in Kubernetes","image":"https://learnk8s.io/a/4f9741344d5b7e302652468bc6be6b59.png","author":{"@type":"Person","name":"Daniele Polencic"},"publisher":{"@type":"Organization","name":"Learnk8s","logo":{"@type":"ImageObject","url":"https://learnk8s.io/a/7489aa207ee6e110df66bfcd9f741652.png"}},"url":"https://learnk8s.io/kubernetes-long-lived-connections","datePublished":"2020-02-05","dateModified":"2020-02-05","description":"Kubernetes doesn't load balance long-lived connections, and some Pods might receive more requests than others. If you're using HTTP/2, gRPC, RSockets, AMQP or any other long-lived connection such as a database connection, you might want to consider client-side load balancing.","mainEntityOfPage":{"@type":"SoftwareSourceCode"}}</script><style>@media screen and (max-width:64em){#main-menu .reveal{display:none;background-color:#001b44}#main-menu:target .reveal{display:inherit}}.hamburger-icon{position:relative;display:inline-block;width:1.625em;height:.213em;background:#fff;border-radius:.2rem}.hamburger-icon:after,.hamburger-icon:before{content:"";position:absolute;width:1.625em;height:.213em;background:#fff}.hamburger-icon:before{top:-.625em}.hamburger-icon:after{bottom:-.625em}.x-icon{position:relative;display:inline-block;width:1.625em;height:.213em}.x-icon:after,.x-icon:before{content:"";position:absolute;top:50%;left:0;margin-top:-.106em;width:1.625em;height:.213em;background:#fff}.x-icon:before{transform:rotate(45deg)}.x-icon:after{transform:rotate(-45deg)}input[type=radio]:checked~.checked-reveal{display:block}.pagination-icon{stroke:currentColor;stroke-linecap:round;stroke-linejoin:round;stroke-width:.125rem;display:inline-block;width:.4rem}html{line-height:1.15;-webkit-text-size-adjust:100%}body{margin:0}h1{margin:.67em 0}hr{box-sizing:content-box;height:0;overflow:visible}pre{font-family:monospace,monospace;font-size:1em}a{background-color:transparent}code{font-size:1em}img{border-style:none}button,input,textarea{font-family:inherit;font-size:100%;line-height:1.15;margin:0}button,input{overflow:visible}button{text-transform:none}[type=submit]{-webkit-appearance:button}textarea{overflow:auto}[type=radio]{box-sizing:border-box;padding:0}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}a,article,blockquote,body,code,div,footer,form,h1,h2,header,html,input[type=email],input[type=text],li,nav,ol,p,pre,section,textarea,ul{box-sizing:border-box}.bg-light-gray{background-color:#eee}.bg-near-white{background-color:#f4f4f4}.white{color:#fff}.bg-white{background-color:#fff}.black-90{color:rgba(0,0,0,.9)}.black-80{color:rgba(0,0,0,.8)}.b--black-70{border-color:rgba(0,0,0,.7)}.black-60{color:rgba(0,0,0,.6)}.black-50{color:rgba(0,0,0,.5)}.bg-black-10{background-color:rgba(0,0,0,.1)}.white-70{color:hsla(0,0%,100%,.7)}.bg-dark-red{background-color:#e7040f}.bg-yellow{background-color:gold}.bg-green{background-color:#19a974}.bg-blue{background-color:#357edd}.b--blue{border-color:#357edd}.b--light-blue{border-color:#96ccff}.navy{color:#001b44}.b--navy{border-color:#001b44}.hover-sky:focus,.hover-sky:hover,.sky{color:#569ad1}.bg-sky{background-color:#569ad1}.bg-evian{background-color:#f7f9fc}.link{text-decoration:none}.link,.link:active,.link:focus,.link:hover,.link:link,.link:visited{transition:color .15s ease-in}.link:focus{outline:1px dotted currentColor}.aspect-ratio{height:0;position:relative}.aspect-ratio--6x4{padding-bottom:66.6%}.aspect-ratio--1x1{padding-bottom:100%}.aspect-ratio--object{position:absolute;top:0;right:0;bottom:0;left:0;width:100%;height:100%;z-index:100}.bt{border-top-style:solid;border-top-width:1px}.bb{border-bottom-style:solid;border-bottom-width:1px}.bl{border-left-style:solid;border-left-width:1px}.bn{border-style:none;border-width:0}.br1{border-radius:.125rem}.br2{border-radius:.25rem}.br-100{border-radius:100%}.br--bottom{border-top-left-radius:0;border-top-right-radius:0}.br--top{border-bottom-right-radius:0}.br--top{border-bottom-left-radius:0}.b--none{border-style:none}.bw1{border-width:.125rem}.bw2{border-width:.25rem}.top-0{top:0}.right-0{right:0}.bottom-0{bottom:0}.left-0{left:0}.bottom-1{bottom:1rem}.cf:after,.cf:before{content:" ";display:table}.cf:after{clear:both}.dn{display:none}.db{display:block}.dib{display:inline-block}.flex{display:flex}.flex-wrap{flex-wrap:wrap}.items-start{align-items:flex-start}.items-center{align-items:center}.justify-between{justify-content:space-between}.i{font-style:italic}.fs-normal{font-style:normal}.b{font-weight:700}.h1{height:1rem}.h2{height:2rem}.center{margin-left:auto}.center{margin-right:auto}.lh-copy{line-height:1.5}.mw4{max-width:8rem}.mw6{max-width:32rem}.mw7{max-width:48rem}.mw8{max-width:64rem}.w1{width:1rem}.w2{width:2rem}.w3{width:4rem}.w4{width:8rem}.w5{width:16rem}.w-20{width:20%}.w-40{width:40%}.w-60{width:60%}.w-100{width:100%}.overflow-hidden{overflow:hidden}.overflow-auto{overflow:auto}.relative{position:relative}.absolute{position:absolute}.pl0{padding-left:0}.pt0{padding-top:0}.mt0{margin-top:0}.mb0,.mv0{margin-bottom:0}.mv0{margin-top:0}.pa1{padding:.25rem}.pl1{padding-left:.25rem}.pb1,.pv1{padding-bottom:.25rem}.pv1{padding-top:.25rem}.ph1{padding-left:.25rem;padding-right:.25rem}.ml1{margin-left:.25rem}.mb1,.mv1{margin-bottom:.25rem}.mv1{margin-top:.25rem}.pa2{padding:.5rem}.pt2{padding-top:.5rem}.pb2,.pv2{padding-bottom:.5rem}.pv2{padding-top:.5rem}.ph2{padding-left:.5rem;padding-right:.5rem}.ml2{margin-left:.5rem}.mt2{margin-top:.5rem}.mb2,.mv2{margin-bottom:.5rem}.mv2{margin-top:.5rem}.mh2{margin-left:.5rem;margin-right:.5rem}.pt3{padding-top:1rem}.pb3,.pv3{padding-bottom:1rem}.pv3{padding-top:1rem}.ph3{padding-left:1rem;padding-right:1rem}.mb3,.mv3{margin-bottom:1rem}.mv3{margin-top:1rem}.pt4{padding-top:2rem}.pb4,.pv4{padding-bottom:2rem}.pv4{padding-top:2rem}.ph4{padding-left:2rem;padding-right:2rem}.mr4{margin-right:2rem}.mt4{margin-top:2rem}.mb4,.mv4{margin-bottom:2rem}.mv4{margin-top:2rem}.underline{text-decoration:underline}.no-underline{text-decoration:none}.tr{text-align:right}.tc{text-align:center}.ttu{text-transform:uppercase}.f2{font-size:2.25rem}.f3{font-size:1.5rem}.f4{font-size:1.25rem}.f5{font-size:1rem}.f6{font-size:.875rem}.f7{font-size:.75rem}.measure{max-width:30em}.measure-wide{max-width:34em}.measure-narrow{max-width:20em}.v-mid{vertical-align:middle}.v-btm{vertical-align:bottom}.o-0{opacity:0}.sans-serif{font-family:-apple-system,BlinkMacSystemFont,avenir next,avenir,helvetica neue,helvetica,ubuntu,roboto,noto,segoe ui,arial,sans-serif}.code{font-family:Consolas,monaco,monospace}.input-reset{-webkit-appearance:none;-moz-appearance:none}.underline-hover:focus,.underline-hover:hover{text-decoration:underline}.pointer:hover,.shadow-hover{cursor:pointer}img{max-width:100%}.list{list-style-type:none}.z-999{z-index:999}[class^=code-]{-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-hyphens:none;-moz-hyphens:none;-ms-hyphens:none;hyphens:none}[class^=code-] .standard{padding-left:2rem;padding-right:2rem;display:block;min-width:100%;box-sizing:border-box}@keyframes blinking{0%,to{color:transparent}50%{color:#fff}}@-moz-keyframes blinking{0%,to{color:transparent}50%{color:#fff}}@-webkit-keyframes blinking{0%,to{color:transparent}50%{color:#fff}}@-ms-keyframes blinking{0%,to{color:transparent}50%{color:#fff}}@-o-keyframes blinking{0%,to{color:transparent}50%{color:#fff}}.code-light-theme{color:#657b83;background-color:#fdf6e3}.code-light-theme::selection,.code-light-theme ::selection{background:#073642}.code-light-theme .token.comment{color:#93a1a1}.code-light-theme .token.punctuation{color:#586e75}.code-light-theme .token.string{color:#2aa198}.code-light-theme .token.keyword{color:#859900}.code-light-theme .token.function{color:#b58900}@media screen and (min-width:32em){.db-ns{display:block}.flex-ns{display:flex}.items-start-ns{align-items:flex-start}.justify-around-ns{justify-content:space-around}.w-50-ns{width:50%}.pt0-ns{padding-top:0}.ph4-ns{padding-left:2rem;padding-right:2rem}.pt5-ns{padding-top:4rem}.mb5-ns{margin-bottom:4rem}.f1-ns{font-size:3rem}}@media screen and (min-width:32em) and (max-width:64em){.pr3-m{padding-right:1rem}}@media screen and (min-width:64em){.bn-l{border-style:none;border-width:0}.db-l{display:block}.flex-l{display:flex}.items-center-l{align-items:center}.mw9-l{max-width:96rem}.w-100-l{width:100%}.static-l{position:static}.mr0-l{margin-right:0}.ml3-l{margin-left:1rem}.mb5-l,.mv5-l{margin-bottom:4rem}.mv5-l{margin-top:4rem}.pr7-l{padding-right:16rem}}</style><link rel=icon type=image/png sizes=32x32 href=><style>.sf-hidden{display:none!important}</style><body class="bg-near-white sans-serif"><noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-5WCKPRL" height="0" width="0" style="display: none; visibility: hidden"></iframe></noscript><div class="cf w-100 mw9-l center bg-white"><div class="white mb4 mb5-ns"></div><div class="tc mb4 db mw4 center"><div><a href=https://linkedin.com/in/danielepolencic title="Daniele Polencic" class=link><div class="center w3 mb1"><div class="aspect-ratio aspect-ratio--1x1"></div></div><span class="black-50 f6 db">Daniele Polencic</span></a></div></div><article class="lazy-article ph3 pt0 pb4 mw7 center"><h1 class="navy tc f2 f1-ns">Load balancing and scaling long-lived connections in Kubernetes</h1><p class="f7 black-60 tc ttu">Published in February 2020<hr class="pv2 bn"><hr class="w3 center b--navy mv4 mb5-ns"><blockquote class="pl3 mh2 bl bw2 b--blue bg-evian pv1 ph4"><p class="lh-copy measure-wide f4"><strong class=b>TL;DR:</strong> Kubernetes doesn't load balance long-lived connections, and some Pods might receive more requests than others. If you're using HTTP/2, gRPC, RSockets, AMQP or any other long-lived connection such as a database connection, you might want to consider client-side load balancing.</p></blockquote><p class="lh-copy measure-wide f4">Kubernetes offers two convenient abstractions to deploy apps: Services and Deployments.<p class="lh-copy measure-wide f4">Deployments describe a recipe for what kind and how many copies of your app should run at any given time.<p class="lh-copy measure-wide f4">Each app is deployed as a Pod, and an IP address is assigned to it.<p class="lh-copy measure-wide f4">Services, on the other hand, are similar to load balancers.<p class="lh-copy measure-wide f4">They are designed to distribute the traffic to a set of Pods.<div class="relative overflow-hidden"><ul class="pl0 list"><li class=mv3><input type=radio name=carousel-1 id=carousel-1-0 checked class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal"><img src="" alt="In this diagram you have three instances of a single app and a load balancer." loading=lazy><div class="bt b-solid bw2 b--black-70 relative mt0"><div class="bg-black-10 br1 pa1 dib mt2 absolute bottom-1 left-0"><span class="b black-60">1</span><span class="f7 black-50">/4</span></div></div><div class="flex items-start justify-between bg-evian ph2"><div class=w-20></div><div class="f5 lh-copy black-90 w-60 center"><p class="lh-copy measure-wide f5">In this diagram you have three instances of a single app and a load balancer.</p></div><label class="db f6 b black-50 pv3 pointer w-20 tr ttu" for=carousel-1-1>Next <svg viewBox="0 0 10 16" xmlns=http://www.w3.org/2000/svg class="pagination-icon ml2"><polyline fill=none vector-effect=non-scaling-stroke points="2,2 8,8 2,14"></polyline></svg></label></div></div><li class=mv3><input type=radio name=carousel-1 id=carousel-1-1 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="The load balancer is called Service and has an IP address. Any incoming request is distributed to one of the Pods." loading=lazy data-single-file-origin-u-r-l=/a/f1bb54a36e6c7b5c40bd854ac58de531.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div><li class=mv3><input type=radio name=carousel-1 id=carousel-1-2 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="The Deployment defines a recipe to create more instances of the same Pod. You almost never deploy Pod individually." loading=lazy data-single-file-origin-u-r-l=/a/556484ee82220fdcf045674f9d5a7b96.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div><li class=mv3><input type=radio name=carousel-1 id=carousel-1-3 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="Pods have IP address assigned to them." loading=lazy data-single-file-origin-u-r-l=/a/6f0a80390c410824b6ec643b2836583e.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div></ul></div><p class="lh-copy measure-wide f4">It is often useful to think about Services as a collection of IP address.<p class="lh-copy measure-wide f4">Every time you make a request to a Service, one of the IP addresses from that list is selected and used as the destination.<div class="relative overflow-hidden"><ul class="pl0 list"><li class=mv3><input type=radio name=carousel-2 id=carousel-2-0 checked class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal"><img src="" alt="Imagine issuing a request such as curl 10.96.45.152 to the Service." loading=lazy><div class="bt b-solid bw2 b--black-70 relative mt0"><div class="bg-black-10 br1 pa1 dib mt2 absolute bottom-1 left-0"><span class="b black-60">1</span><span class="f7 black-50">/3</span></div></div><div class="flex items-start justify-between bg-evian ph2"><div class=w-20></div><div class="f5 lh-copy black-90 w-60 center"><p class="lh-copy measure-wide f5">Imagine issuing a request such as <code class="code f5 lh-copy bg-near-white br2 pv1 ph2 fs-normal">curl 10.96.45.152</code> to the Service.</p></div><label class="db f6 b black-50 pv3 pointer w-20 tr ttu" for=carousel-2-1>Next <svg viewBox="0 0 10 16" xmlns=http://www.w3.org/2000/svg class="pagination-icon ml2"><polyline fill=none vector-effect=non-scaling-stroke points="2,2 8,8 2,14"></polyline></svg></label></div></div><li class=mv3><input type=radio name=carousel-2 id=carousel-2-1 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="The Service picks one of the three Pods as the destination." loading=lazy data-single-file-origin-u-r-l=/a/56f3c40af81c07a61db3c2f6b7556e4a.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div><li class=mv3><input type=radio name=carousel-2 id=carousel-2-2 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="The traffic is forwarded to that instance." loading=lazy data-single-file-origin-u-r-l=/a/4c7f2ff2809892f2777c5dd505d1a97c.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div></ul></div><p class="lh-copy measure-wide f4">If you have two apps such as a front-end and a backend, you can use a Deployment and a Service for each and deploy them in the cluster.<p class="lh-copy measure-wide f4">When the front-end app makes a request, it doesn't need to know how many Pods are connected to the backend Service.<p class="lh-copy measure-wide f4">It could be one Pod, tens or hundreds.<p class="lh-copy measure-wide f4">The front-end app isn't aware of the individual IP addresses of the backend app either.<p class="lh-copy measure-wide f4">When it wants to make a request, that request is sent to the backend Service which has an IP address that doesn't change.<div class="relative overflow-hidden"><ul class="pl0 list"><li class=mv3><input type=radio name=carousel-3 id=carousel-3-0 checked class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal"><img src="" alt="The red Pod issues a request to an internal (beige) component. Instead of choosing one of the Pod as the destination, the red Pod issues the request to the Service." loading=lazy><div class="bt b-solid bw2 b--black-70 relative mt0"><div class="bg-black-10 br1 pa1 dib mt2 absolute bottom-1 left-0"><span class="b black-60">1</span><span class="f7 black-50">/4</span></div></div><div class="flex items-start justify-between bg-evian ph2"><div class=w-20></div><div class="f5 lh-copy black-90 w-60 center"><p class="lh-copy measure-wide f5">The red Pod issues a request to an internal (beige) component. Instead of choosing one of the Pod as the destination, the red Pod issues the request to the Service.</p></div><label class="db f6 b black-50 pv3 pointer w-20 tr ttu" for=carousel-3-1>Next <svg viewBox="0 0 10 16" xmlns=http://www.w3.org/2000/svg class="pagination-icon ml2"><polyline fill=none vector-effect=non-scaling-stroke points="2,2 8,8 2,14"></polyline></svg></label></div></div><li class=mv3><input type=radio name=carousel-3 id=carousel-3-1 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="The Service selects one of the ready Pods as the destination." loading=lazy data-single-file-origin-u-r-l=/a/b12238547c66d614d73cb3ca871f96d0.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div><li class=mv3><input type=radio name=carousel-3 id=carousel-3-2 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="The traffic flows from the red Pod to the light brown Pod." loading=lazy data-single-file-origin-u-r-l=/a/4cf037dfc2b8cf313786e405c7a6033d.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div><li class=mv3><input type=radio name=carousel-3 id=carousel-3-3 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="Notice how the red Pod doesn't know how many Pods are hidden behind the Service." loading=lazy data-single-file-origin-u-r-l=/a/a2fa7240072613c26d7178d6c79e24ce.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div></ul></div><p class="lh-copy measure-wide f4"><em class=i>But what's the load balancing strategy for the Service?</em><p class="lh-copy measure-wide f4"><em class=i>It is round-robin, right?</em><p class="lh-copy measure-wide f4">Sort of.<h2 class="f2 pt4 pb2" id=load-balancing-in-kubernetes-services>Load balancing in Kubernetes Services</h2><p class="lh-copy measure-wide f4">Kubernetes Services don't exist.<p class="lh-copy measure-wide f4">There's no process listening on the IP address and port of the Service.<blockquote class="pl3 mh2 bl bw2 b--blue bg-evian pv1 ph4"><p class="lh-copy measure-wide f4">You can check that this is the case by accessing any node in your Kubernetes cluster and executing <code class="code f5 lh-copy bg-near-white br2 pv1 ph2 fs-normal">netstat -ntlp</code>.</p></blockquote><p class="lh-copy measure-wide f4">Even the IP address can't be found anywhere.<p class="lh-copy measure-wide f4">The IP address for a Service is allocated by the control plane in the controller manager and stored in the database — etcd.<p class="lh-copy measure-wide f4">That same IP address is then used by another component: kube-proxy.<p class="lh-copy measure-wide f4">Kube-proxy reads the list of IP addresses for all Services and writes a collection of iptables rules in every node.<p class="lh-copy measure-wide f4">The rules are meant to say: "if you see this Service IP address, instead rewrite the request and pick one of the Pod as the destination".<p class="lh-copy measure-wide f4">The Service IP address is used only as a placeholder — that's why there is no process listening on the IP address or port.<div class="relative overflow-hidden"><ul class="pl0 list"><li class=mv3><input type=radio name=carousel-4 id=carousel-4-0 checked class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal"><img src= alt="Consider a cluster with three Nodes. Each Node has a Pod deployed." loading=lazy><div class="bt b-solid bw2 b--black-70 relative mt0"><div class="bg-black-10 br1 pa1 dib mt2 absolute bottom-1 left-0"><span class="b black-60">1</span><span class="f7 black-50">/8</span></div></div><div class="flex items-start justify-between bg-evian ph2"><div class=w-20></div><div class="f5 lh-copy black-90 w-60 center"><p class="lh-copy measure-wide f5">Consider a cluster with three Nodes. Each Node has a Pod deployed.</p></div><label class="db f6 b black-50 pv3 pointer w-20 tr ttu" for=carousel-4-1>Next <svg viewBox="0 0 10 16" xmlns=http://www.w3.org/2000/svg class="pagination-icon ml2"><polyline fill=none vector-effect=non-scaling-stroke points="2,2 8,8 2,14"></polyline></svg></label></div></div><li class=mv3><input type=radio name=carousel-4 id=carousel-4-1 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="The beige Pods are part of a Service. Services don't exist, so the diagram has the component grayed out." loading=lazy data-single-file-origin-u-r-l=/a/fe56fad78ea78f20296a7f553b4e3877.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div><li class=mv3><input type=radio name=carousel-4 id=carousel-4-2 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="The red Pod wants to issue a request to the Service and eventually reaches one of the beige Pods." loading=lazy data-single-file-origin-u-r-l=/a/15512cba51dbba9050eabe792bd6869e.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div><li class=mv3><input type=radio name=carousel-4 id=carousel-4-3 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="But Service don't exist. There's no process listening on the Service IP address. How does it work?" loading=lazy data-single-file-origin-u-r-l=/a/4b1d129e106048342c91e49f78edc140.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div><li class=mv3><input type=radio name=carousel-4 id=carousel-4-4 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="Before the request is dispatched from the Node, it is intercepted by iptables rules." loading=lazy data-single-file-origin-u-r-l=/a/4a975406947f8701361e6bec2870df41.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div><li class=mv3><input type=radio name=carousel-4 id=carousel-4-5 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="The iptables rules know that the Service doesn't exist and proceed to replace the IP address of the Service with one of the IP addresses of the Pods beloning to that Service." loading=lazy data-single-file-origin-u-r-l=/a/032cd8e3e2d23d69d17fb56e49bdf8cf.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div><li class=mv3><input type=radio name=carousel-4 id=carousel-4-6 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="The request has a real IP address as the destination and it can proceed normally." loading=lazy data-single-file-origin-u-r-l=/a/dbb2baf7a34be504a5cec5ef9fb529dd.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div><li class=mv3><input type=radio name=carousel-4 id=carousel-4-7 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="Depening on your particular network implementation, the request finally reaches the Pod." loading=lazy data-single-file-origin-u-r-l=/a/0c4c84a33748362073af2d0908b03e30.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div></ul></div><p class="lh-copy measure-wide f4"><em class=i>Does iptables use round-robin?</em><p class="lh-copy measure-wide f4">No, iptables is primarily used for firewalls, and it is not designed to do load balancing.<p class="lh-copy measure-wide f4">However, you could <a href=https://scalingo.com/blog/iptables#load-balancing target=_blank rel=noreferrer class="link navy underline hover-sky">craft a smart set of rules that could make iptables behave like a load balancer</a>.<p class="lh-copy measure-wide f4">And this is precisely what happens in Kubernetes.<p class="lh-copy measure-wide f4">If you have three Pods, kube-proxy writes the following rules:<ol><li class="lh-copy f4 mv1 measure-wide">select Pod 1 as the destination with a likelihood of 33%. Otherwise, move to the next rule<li class="lh-copy f4 mv1 measure-wide">choose Pod 2 as the destination with a probability of 50%. Otherwise, move to the following rule<li class="lh-copy f4 mv1 measure-wide">select Pod 3 as the destination (no probability)</ol><p class="lh-copy measure-wide f4">The compound probability is that Pod 1, Pod 2 and Pod 3 have all have a one-third chance (33%) to be selected.</p><img class="db pv3 center" src="" alt="iptables rules for three Pods" loading=lazy><p class="lh-copy measure-wide f4">Also, there's no guarantee that Pod 2 is selected after Pod 1 as the destination.<blockquote class="pl3 mh2 bl bw2 b--blue bg-evian pv1 ph4"><p class="lh-copy measure-wide f4">Iptables use the <a href=http://ipset.netfilter.org/iptables-extensions.man.html#lbCD target=_blank rel=noreferrer class="link navy underline hover-sky">statistic module</a> with <code class="code f5 lh-copy bg-near-white br2 pv1 ph2 fs-normal">random</code> mode. So the load balancing algorithm is random.</p></blockquote><p class="lh-copy measure-wide f4">Now that you're familiar with how Services work let's have a look at more exciting scenarios.<h2 class="f2 pt4 pb2" id=long-lived-connections-don-t-scale-out-of-the-box-in-kubernetes>Long-lived connections don't scale out of the box in Kubernetes</h2><p class="lh-copy measure-wide f4">With every HTTP request started from the front-end to the backend, a new TCP connection is opened and closed.<p class="lh-copy measure-wide f4">If the front-end makes 100 HTTP requests per second to the backend, 100 different TCP connections are opened and closed in that second.<p class="lh-copy measure-wide f4">You can improve the latency and save resources if you open a TCP connection and reuse it for any subsequent HTTP requests.<p class="lh-copy measure-wide f4">The HTTP protocol has a feature called HTTP keep-alive, or HTTP connection reuse that uses a single TCP connection to send and receive multiple HTTP requests and responses.</p><img class="db pv3 center" src= alt="Opening and closing connections VS HTTP connection reuse" loading=lazy><p class="lh-copy measure-wide f4">It doesn't work out of the box; your server and client should be configured to use it.<p class="lh-copy measure-wide f4">The change itself is straightforward, and it's available in most languages and frameworks.<p class="lh-copy measure-wide f4">Here a few examples on how to implement keep-alive in different languages:<ul><li class="lh-copy f4 mv1 measure-wide"><a href=https://medium.com/@onufrienkos/keep-alive-connection-on-inter-service-http-requests-3f2de73ffa1 target=_blank rel=noreferrer class="link navy underline hover-sky">Keep-alive in Node.js</a><li class="lh-copy f4 mv1 measure-wide"><a href=https://www.baeldung.com/httpclient-connection-management target=_blank rel=noreferrer class="link navy underline hover-sky">Keep-alive in Spring boot</a><li class="lh-copy f4 mv1 measure-wide"><a href=https://blog.insightdatascience.com/learning-about-the-http-connection-keep-alive-header-7ebe0efa209d target=_blank rel=noreferrer class="link navy underline hover-sky">Keep-alive in Python</a><li class="lh-copy f4 mv1 measure-wide"><a href="https://docs.microsoft.com/en-us/dotnet/api/system.net.httpwebrequest.keepalive?view=netframework-4.8" target=_blank rel=noreferrer class="link navy underline hover-sky">Keep-alive in .NET</a></ul><p class="lh-copy measure-wide f4"><em class=i>What happens when you use keep-alive with a Kubernetes Service?</em><p class="lh-copy measure-wide f4">Let's imagine that front-end and backend support keep-alive.<p class="lh-copy measure-wide f4">You have a single instance of the front-end and three replicas for the backend.<p class="lh-copy measure-wide f4">The front-end makes the first request to the backend and opens the TCP connection.<p class="lh-copy measure-wide f4">The request reaches the Service, and one of the Pod is selected as the destination.<p class="lh-copy measure-wide f4">The backend Pod replies and the front-end receives the response.<p class="lh-copy measure-wide f4">But instead of closing the TCP connection, it is kept open for subsequent HTTP requests.<p class="lh-copy measure-wide f4"><em class=i>What happens when the front-end issues more requests?</em><p class="lh-copy measure-wide f4">They are sent to the same Pod.<p class="lh-copy measure-wide f4"><em class=i>Isn't iptables supposed to distribute the traffic?</em><p class="lh-copy measure-wide f4">It is.<p class="lh-copy measure-wide f4">There is a single TCP connection open, and iptables rule were invocated the first time.<p class="lh-copy measure-wide f4">One of the three Pods was selected as the destination.<p class="lh-copy measure-wide f4">Since all subsequent requests are channelled through the same TCP connection, <a href=https://scalingo.com/blog/iptables#load-balancing target=_blank rel=noreferrer class="link navy underline hover-sky">iptables isn't invoked anymore.</a><div class="relative overflow-hidden"><ul class="pl0 list"><li class=mv3><input type=radio name=carousel-5 id=carousel-5-0 checked class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal"><img src="" alt="The red Pod issues a request to the Service." loading=lazy><div class="bt b-solid bw2 b--black-70 relative mt0"><div class="bg-black-10 br1 pa1 dib mt2 absolute bottom-1 left-0"><span class="b black-60">1</span><span class="f7 black-50">/5</span></div></div><div class="flex items-start justify-between bg-evian ph2"><div class=w-20></div><div class="f5 lh-copy black-90 w-60 center"><p class="lh-copy measure-wide f5">The red Pod issues a request to the Service.</p></div><label class="db f6 b black-50 pv3 pointer w-20 tr ttu" for=carousel-5-1>Next <svg viewBox="0 0 10 16" xmlns=http://www.w3.org/2000/svg class="pagination-icon ml2"><polyline fill=none vector-effect=non-scaling-stroke points="2,2 8,8 2,14"></polyline></svg></label></div></div><li class=mv3><input type=radio name=carousel-5 id=carousel-5-1 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="You already know what happens next. Services don't exist, but iptables rules intercept the requests." loading=lazy data-single-file-origin-u-r-l=/a/4a975406947f8701361e6bec2870df41.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div><li class=mv3><input type=radio name=carousel-5 id=carousel-5-2 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="One of the Pods that belong the Service is selected as the destination." loading=lazy data-single-file-origin-u-r-l=/a/032cd8e3e2d23d69d17fb56e49bdf8cf.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div><li class=mv3><input type=radio name=carousel-5 id=carousel-5-3 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="Finally the request reaches the Pod. At this point, a persistent connection between the two Pods is established." loading=lazy data-single-file-origin-u-r-l=/a/c0f31b65b8a7ad76eaf1d8a72e441124.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div><li class=mv3><input type=radio name=carousel-5 id=carousel-5-4 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="Any subsequet request from the red Pod reuses the existing open connection." loading=lazy data-single-file-origin-u-r-l=/a/ca327c2c19bc23c2a7b7b1f9d883c8e1.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div></ul></div><p class="lh-copy measure-wide f4">So you have now achieved better latency and throughput, but you lost the ability to scale your backend.<p class="lh-copy measure-wide f4">Even if you have two backend Pods that can receive requests from the frontend Pod, only one is actively used.<p class="lh-copy measure-wide f4"><em class=i>Is it fixable?</em><p class="lh-copy measure-wide f4">Since Kubernetes doesn't know how to load balance persistent connections, you could step in and fix it yourself.<p class="lh-copy measure-wide f4">Services are a collection of IP addresses and ports — usually called endpoints.<p class="lh-copy measure-wide f4">Your app could retrieve the list of endpoints from the Service and decide how to distribute the requests.<p class="lh-copy measure-wide f4">As a first try, you could open a persistent connection to every Pod and round-robin requests to them.<p class="lh-copy measure-wide f4">Or you could <a href=https://blog.twitter.com/engineering/en_us/topics/infrastructure/2019/daperture-load-balancer.html target=_blank rel=noreferrer class="link navy underline hover-sky">implement more sophisticated load balancing algorithms</a>.<p class="lh-copy measure-wide f4">The client-side code that executes the load balancing should follow the logic below:<ol><li class="lh-copy f4 mv1 measure-wide">retrieve a list of endpoints from the Service<li class="lh-copy f4 mv1 measure-wide">for each of them, open a connection and keep it open<li class="lh-copy f4 mv1 measure-wide">when you need to make a request, pick one of the open connections<li class="lh-copy f4 mv1 measure-wide">on a regular interval refresh the list of endpoints and remove or add new connections</ol><div class="relative overflow-hidden"><ul class="pl0 list"><li class=mv3><input type=radio name=carousel-6 id=carousel-6-0 checked class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal"><img src="" alt="Instead of having the red Pod issuing a request to your Service, you could load balance the request client-side." loading=lazy><div class="bt b-solid bw2 b--black-70 relative mt0"><div class="bg-black-10 br1 pa1 dib mt2 absolute bottom-1 left-0"><span class="b black-60">1</span><span class="f7 black-50">/4</span></div></div><div class="flex items-start justify-between bg-evian ph2"><div class=w-20></div><div class="f5 lh-copy black-90 w-60 center"><p class="lh-copy measure-wide f5">Instead of having the red Pod issuing a request to your Service, you could load balance the request client-side.</p></div><label class="db f6 b black-50 pv3 pointer w-20 tr ttu" for=carousel-6-1>Next <svg viewBox="0 0 10 16" xmlns=http://www.w3.org/2000/svg class="pagination-icon ml2"><polyline fill=none vector-effect=non-scaling-stroke points="2,2 8,8 2,14"></polyline></svg></label></div></div><li class=mv3><input type=radio name=carousel-6 id=carousel-6-1 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="You could write some code that asks what Pods are part of the Service." loading=lazy data-single-file-origin-u-r-l=/a/16a3c4cb30f9c6d0fbfcd628facf42d5.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div><li class=mv3><input type=radio name=carousel-6 id=carousel-6-2 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="Once you have that list, you could store it locally and use it to connect to the Pods." loading=lazy data-single-file-origin-u-r-l=/a/0475611802c11a71cfbe90e355af6e0f.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div><li class=mv3><input type=radio name=carousel-6 id=carousel-6-3 class="o-0 absolute bottom-0 left-0"><div class="dn checked-reveal sf-hidden"><img src="" alt="You are in charge of the load balancing algorithm." loading=lazy data-single-file-origin-u-r-l=/a/987df827edd7bdd73ebbc4b1d313b13f.svg><div class="bt b-solid bw2 b--black-70 relative mt0"></div><div class="flex items-start justify-between bg-evian ph2"></div></div></ul></div><p class="lh-copy measure-wide f4"><em class=i>Does this problem apply only to HTTP keep-alive?</em><h2 class="f2 pt4 pb2" id=client-side-load-balancing>Client-side load balancing</h2><p class="lh-copy measure-wide f4">HTTP isn't the only protocol that can benefit from long-lived TCP connections.<p class="lh-copy measure-wide f4">If your app uses a database, the connection isn't opened and closed every time you wish to retrieve a record or a document.<p class="lh-copy measure-wide f4">Instead, the TCP connection is established once and kept open.<p class="lh-copy measure-wide f4">If your database is deployed in Kubernetes using a Service, you might experience the same issues as the previous example.<p class="lh-copy measure-wide f4">There's one replica in your database that is utilised more than the others.<p class="lh-copy measure-wide f4">Kube-proxy and Kubernetes don't help to balance persistent connections.<p class="lh-copy measure-wide f4">Instead, you should take care of load balancing the requests to your database.<p class="lh-copy measure-wide f4">Depending on the library that you use to connect to the database, you might have different options.<p class="lh-copy measure-wide f4">The following example is from a clustered MySQL database called from Node.js:<div class="mv4 mv5-l"><header class="bg-light-gray flex pv2 pl1 br--top br2 relative"><div class="w1 h1 ml1 br-100 bg-dark-red"></div><div class="w1 h1 ml1 br-100 bg-green"></div><div class="w1 h1 ml1 br-100 bg-yellow"></div><p class="code f6 mv0 black-60 w-100 tc absolute top-0 left-0 h1 pv2">index.js</p></header><pre class="code-light-theme relative pv4 overflow-auto mv0 br2 br--bottom"><code class="code lh-copy"><span class=standard><span class="token keyword">var</span> mysql <span class="token operator">=</span> <span class="token function">require</span><span class="token punctuation">(</span><span class="token string">'mysql'</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">var</span> poolCluster <span class="token operator">=</span> mysql<span class="token punctuation">.</span><span class="token function">createPoolCluster</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

<span class="token keyword">var</span> endpoints <span class="token operator">=</span> <span class="token comment">/* retrieve endpoints from the Service */</span>

<span class="token keyword">for</span> <span class="token punctuation">(</span><span class="token keyword">var</span> <span class="token punctuation">[</span>index<span class="token punctuation">,</span> endpoint<span class="token punctuation">]</span> <span class="token keyword">of</span> endpoints<span class="token punctuation">)</span> <span class="token punctuation">{</span>
  poolCluster<span class="token punctuation">.</span><span class="token function">add</span><span class="token punctuation">(</span><span class="token template-string"><span class="token string">`mysql-replica-</span><span class="token interpolation"><span class="token interpolation-punctuation punctuation">${</span>index<span class="token interpolation-punctuation punctuation">}</span></span><span class="token string">`</span></span><span class="token punctuation">,</span> endpoint<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>

<span class="token comment">// Make queries to the clustered MySQL database</span></span></code></pre></div><p class="lh-copy measure-wide f4">As you can imagine, several other protocols work over long-lived TCP connections.<p class="lh-copy measure-wide f4">Here you can read a few examples:<ul><li class="lh-copy f4 mv1 measure-wide">Websockets and secured WebSockets<li class="lh-copy f4 mv1 measure-wide">HTTP/2<li class="lh-copy f4 mv1 measure-wide">gRPC<li class="lh-copy f4 mv1 measure-wide">RSockets<li class="lh-copy f4 mv1 measure-wide">AMQP</ul><p class="lh-copy measure-wide f4">You might recognise most of the protocols above.<p class="lh-copy measure-wide f4"><em class=i>So if these protocols are so popular, why isn't there a standard answer to load balancing?</em><p class="lh-copy measure-wide f4"><em class=i>Why does the logic have to be moved into the client?</em><p class="lh-copy measure-wide f4"><em class=i>Is there a native solution in Kubernetes?</em><p class="lh-copy measure-wide f4">Kube-proxy and iptables are designed to cover the most popular use cases of deployments in a Kubernetes cluster.<p class="lh-copy measure-wide f4">But they are mostly there for convenience.<p class="lh-copy measure-wide f4">If you're using a web service that exposes a REST API, then you're in luck — this use case usually doesn't reuse TCP connections, and you can use any Kubernetes Service.<p class="lh-copy measure-wide f4">But as soon as you start using persistent TCP connections, you should look into how you can evenly distribute the load to your backends.<p class="lh-copy measure-wide f4">Kubernetes doesn't cover that specific use case out of the box.<p class="lh-copy measure-wide f4">However, there's something that could help.<h2 class="f2 pt4 pb2" id=load-balancing-long-lived-connections-in-kubernetes>Load balancing long-lived connections in Kubernetes</h2><p class="lh-copy measure-wide f4">Kubernetes has four different kinds of Services:<ol><li class="lh-copy f4 mv1 measure-wide">ClusterIP<li class="lh-copy f4 mv1 measure-wide">NodePort<li class="lh-copy f4 mv1 measure-wide">LoadBalancer<li class="lh-copy f4 mv1 measure-wide">Headless</ol><p class="lh-copy measure-wide f4">The first three Services have a virtual IP address that is used by kube-proxy to create iptables rules.<p class="lh-copy measure-wide f4">But the fundamental building block of all kinds of the Services is the Headless Service.<p class="lh-copy measure-wide f4">The headless Service doesn't have an assigned IP address and is only a mechanism to collect a list of Pod IP addresses and ports (also called endpoints).<p class="lh-copy measure-wide f4">Every other Service is built on top of the Headless Service.<p class="lh-copy measure-wide f4">The ClusterIP Service is a Headless Service with some extra features:<ul><li class="lh-copy f4 mv1 measure-wide">the control plane assigns it an IP address<li class="lh-copy f4 mv1 measure-wide">kube-proxy iterates through all the IP addresses and creates iptables rules</ul><p class="lh-copy measure-wide f4">So you could ignore kube-proxy all together and always use the list of endpoints collected by the Headless Service to load balance requests client-side.<p class="lh-copy measure-wide f4"><em class=i>But can you imagine adding that logic to all apps deployed in the cluster?</em><p class="lh-copy measure-wide f4">If you have an existing fleet of applications, this might sound like an impossible task.<p class="lh-copy measure-wide f4">But there's an alternative.<h2 class="f2 pt4 pb2" id=service-meshes-to-the-rescue>Service meshes to the rescue</h2><p class="lh-copy measure-wide f4">You probably already noticed that the client-side load balancing strategy is quite standard.<p class="lh-copy measure-wide f4">When the app starts, it should<ul><li class="lh-copy f4 mv1 measure-wide">retrieve a list of IP addresses from the Service<li class="lh-copy f4 mv1 measure-wide">open and maintain a pool of connections<li class="lh-copy f4 mv1 measure-wide">periodically refresh the pool by adding and removing endpoints</ul><p class="lh-copy measure-wide f4">As soon as it wishes to make a request, it should:<ul><li class="lh-copy f4 mv1 measure-wide">pick one of the available connections using a predefined logic such as round-robin<li class="lh-copy f4 mv1 measure-wide">issue the request</ul><p class="lh-copy measure-wide f4">The steps above are valid for WebSockets connections as well as gRPC and AMQP.<p class="lh-copy measure-wide f4">You could extract that logic in a separate library and share it with all apps.<p class="lh-copy measure-wide f4">Instead of writing a library from scratch, you could use a Service mesh such as Istio or Linkerd.<p class="lh-copy measure-wide f4">Service meshes augment your app with a new process that:<ul><li class="lh-copy f4 mv1 measure-wide">automatically discovers IP addresses Services<li class="lh-copy f4 mv1 measure-wide">inspects connections such as WebSockets and gRPC<li class="lh-copy f4 mv1 measure-wide">load-balances requests using the right protocol</ul><p class="lh-copy measure-wide f4">Service meshes can help you to manage the traffic inside your cluster, but they aren't exactly lightweight.<p class="lh-copy measure-wide f4">Other options include using a library such as Netflix Ribbon, a programmable proxy such as Envoy or just ignore it.<p class="lh-copy measure-wide f4"><em class=i>What happens if you ignore it?</em><p class="lh-copy measure-wide f4">You can ignore the load balancing and still don't notice any change.<p class="lh-copy measure-wide f4">There are a couple of scenarios that you should consider.<p class="lh-copy measure-wide f4">If you have more clients than servers, there should be limited issues.<p class="lh-copy measure-wide f4">Imagine you have five clients opening persistent connections to two servers.<p class="lh-copy measure-wide f4">Even if there's no load balancing, both servers likely utilised.</p><img class="db pv3 center" src="" alt="More clients than servers" loading=lazy><p class="lh-copy measure-wide f4">The connections might not be distributed evenly (perhaps four ended up connecting to the same server), but overall there's a good chance that both servers are utilised.<p class="lh-copy measure-wide f4">What's more problematic is the opposite scenario.<p class="lh-copy measure-wide f4">If you have fewer clients and more servers, you might have some underutilised resources and a potential bottleneck.<p class="lh-copy measure-wide f4">Imagine having two clients and five servers.<p class="lh-copy measure-wide f4">At best, two persistent connections to two servers are opened.<p class="lh-copy measure-wide f4">The remaining servers are not used at all.</p><img class="db pv3 center" src="" alt="More servers than clients" loading=lazy><p class="lh-copy measure-wide f4">If the two servers can't handle the traffic generated by the clients, horizontal scaling won't help.<h2 class="f2 pt4 pb2" id=summary>Summary</h2><p class="lh-copy measure-wide f4">Kubernetes Services are designed to cover most common uses for web applications.<p class="lh-copy measure-wide f4">However, as soon as you start working with application protocols that use persistent TCP connections, such as databases, gRPC, or WebSockets, they fall apart.<p class="lh-copy measure-wide f4">Kubernetes doesn't offer any built-in mechanism to load balance long-lived TCP connections.<p class="lh-copy measure-wide f4">Instead, you should code your application so that it can retrieve and load balance upstreams client-side.<p class="lh-copy measure-wide f4">Many thanks to <a href=https://medium.com/@weibeld target=_blank rel=noreferrer class="link navy underline hover-sky">Daniel Weibel</a>, <a href=https://github.com/errge target=_blank rel=noreferrer class="link navy underline hover-sky">Gergely Risko</a> and <a href=https://twitter.com/soulmaniqbal target=_blank rel=noreferrer class="link navy underline hover-sky">Salman Iqbal</a> for offering some invaluable suggestions.<p class="lh-copy measure-wide f4">And to <a href=https://twitter.com/CloudNativChris target=_blank rel=noreferrer class="link navy underline hover-sky">Chris Hanson</a> who suggested to include the detailed explanation (and flow chart) on how iptables rules work in practice.</p></article><div class="mb4 mb5-l mw8 center"><ul class="list pl0 flex flex-wrap"></ul></div></div><script data-template-shadow-root>(() => { document.currentScript.remove(); const processNode = node => { node.querySelectorAll("template[shadowmode]").forEach(element=>{ if (!element.parentElement.shadowRoot) { const shadowRoot = element.parentElement.attachShadow({mode:element.getAttribute("shadowmode"),delegatesFocus:Boolean(element.getAttribute("delegatesfocus"))}); shadowRoot.innerHTML = element.innerHTML; element.remove(); processNode(shadowRoot); } }) }; const FORBIDDEN_TAG_NAMES = ["a","area","audio","base","br","col","command","embed","hr","img","iframe","input","keygen","link","meta","param","source","track","video","wbr"]; const NOTE_TAGNAME = "single-file-note"; const NOTE_CLASS = "note"; const NOTE_ANCHORED_CLASS = "note-anchored"; const NOTE_SELECTED_CLASS = "note-selected"; const NOTE_MOVING_CLASS = "note-moving"; const NOTE_MASK_MOVING_CLASS = "note-mask-moving"; const MASK_CLASS = "single-file-mask"; const HIGHLIGHT_CLASS = "single-file-highlight"; const NOTES_WEB_STYLESHEET = ".note { all: initial; display: flex; flex-direction: column; height: 150px; width: 150px; position: absolute; top: 10px; left: 10px; border: 1px solid rgb(191, 191, 191); z-index: 2147483646; box-shadow: 2px 2px 5px rgba(33, 33, 33, .7); min-height: 100px; min-width: 100px; } .note-selected { z-index: 2147483647; } .note-hidden { display: none; } .note-collapsed { min-height: 30px; max-height: 30px; overflow: hidden; } .note textarea { all: initial; white-space: pre; font-family: Arial, Helvetica, sans-serif; font-size: 14px; padding: 3px; height: 100%; border: 1px solid transparent; resize: none; } .note textarea:focus { border: 1px dotted rgb(160, 160, 160); } .note header { all: initial; min-height: 30px; cursor: grab; user-select: none; } .note .note-remove { all: initial; position: absolute; right: 0px; top: 2px; padding: 5px; opacity: .5; cursor: pointer; user-select: none; } .note .note-anchor { all: initial; position: absolute; left: 0px; top: 2px; padding: 5px; opacity: .25; cursor: pointer; } .note .note-resize { all: initial; position: absolute; bottom: -5px; right: -5px; height: 15px; width: 15px; cursor: nwse-resize; user-select: none; } .note .note-remove:hover { opacity: 1; } .note .note-anchor:hover { opacity: .5; } .note-anchored .note-anchor { opacity: .5; } .note-anchored .note-anchor:hover { opacity: 1; } .note-moving { opacity: .5; } .note-moving * { cursor: grabbing; } .note-yellow header { background-color: #f5f545; } .note-yellow textarea { background-color: #ffff7c; } .note-pink header { background-color: #ffa59f; } .note-pink textarea { background-color: #ffbbb6; } .note-blue header { background-color: #84c8ff; } .note-blue textarea { background-color: #95d0ff; } .note-green header { background-color: #93ef8d; } .note-green textarea { background-color: #9cff95; }"; const MASK_WEB_STYLESHEET = ".note-mask { all: initial; position: fixed; z-index: 2147483645; pointer-events: none; } .note-mask-moving { background-color: rgba(236, 234, 234, 0.5); } .page-mask { all: initial; position: fixed; top: 0; left: 0; width: 0; height: 0; z-index: 2147483646; } .page-mask-active { width: 100vw; height: 100vh; }"; const NOTE_HEADER_HEIGHT = 25; const PAGE_MASK_ACTIVE_CLASS = "page-mask-active"; const REMOVED_CONTENT_CLASS = "single-file-removed"; const reflowNotes = function reflowNotes() { document.querySelectorAll(NOTE_TAGNAME).forEach(containerElement => { const noteElement = containerElement.shadowRoot.querySelector("." + NOTE_CLASS); const noteBoundingRect = noteElement.getBoundingClientRect(); const anchorElement = getAnchorElement(containerElement); const anchorBoundingRect = anchorElement.getBoundingClientRect(); const maxX = anchorBoundingRect.x + Math.max(0, anchorBoundingRect.width - noteBoundingRect.width); const minX = anchorBoundingRect.x; const maxY = anchorBoundingRect.y + Math.max(0, anchorBoundingRect.height - NOTE_HEADER_HEIGHT); const minY = anchorBoundingRect.y; let left = parseInt(noteElement.style.getPropertyValue("left")); let top = parseInt(noteElement.style.getPropertyValue("top")); if (noteBoundingRect.x > maxX) { left -= noteBoundingRect.x - maxX; } if (noteBoundingRect.x < minX) { left += minX - noteBoundingRect.x; } if (noteBoundingRect.y > maxY) { top -= noteBoundingRect.y - maxY; } if (noteBoundingRect.y < minY) { top += minY - noteBoundingRect.y; } noteElement.style.setProperty("position", "absolute"); noteElement.style.setProperty("left", left + "px"); noteElement.style.setProperty("top", top + "px"); }); }; const addNoteRef = function addNoteRef(anchorElement, noteId) { const noteRefs = getNoteRefs(anchorElement); noteRefs.push(noteId); setNoteRefs(anchorElement, noteRefs); }; const deleteNoteRef = function deleteNoteRef(containerElement, noteId) { const anchorElement = getAnchorElement(containerElement); const noteRefs = getNoteRefs(anchorElement).filter(noteRefs => noteRefs != noteId); if (noteRefs.length) { setNoteRefs(anchorElement, noteRefs); } else { delete anchorElement.dataset.singleFileNoteRefs; } }; const getNoteRefs = function getNoteRefs(anchorElement) { return JSON.parse("[" + (anchorElement.dataset.singleFileNoteRefs || "") + "]"); }; const setNoteRefs = function setNoteRefs(anchorElement, noteRefs) { anchorElement.dataset.singleFileNoteRefs = noteRefs.toString(); }; const getAnchorElement = function getAnchorElement(containerElement) { return document.querySelector("[data-single-file-note-refs^=" + JSON.stringify(containerElement.dataset.noteId) + "], [data-single-file-note-refs$=" + JSON.stringify(containerElement.dataset.noteId) + "], [data-single-file-note-refs*=" + JSON.stringify("," + containerElement.dataset.noteId + ",") + "]"); }; const getMaskElement = function getMaskElement(className, containerClassName) { let maskElement = document.documentElement.querySelector("." + className); if (!maskElement) { maskElement = document.createElement("div"); const maskContainerElement = document.createElement("div"); if (containerClassName) { maskContainerElement.classList.add(containerClassName); } maskContainerElement.classList.add(MASK_CLASS); const firstNote = document.querySelector(NOTE_TAGNAME); if (firstNote && firstNote.parentElement == document.documentElement) { document.documentElement.insertBefore(maskContainerElement, firstNote); } else { document.documentElement.appendChild(maskContainerElement); } maskElement.classList.add(className); const maskShadow = maskContainerElement.attachShadow({ mode: "open" }); maskShadow.appendChild(getStyleElement(MASK_WEB_STYLESHEET)); maskShadow.appendChild(maskElement); return maskElement; } }; const getStyleElement = function getStyleElement(stylesheet) { const linkElement = document.createElement("style"); linkElement.textContent = stylesheet; return linkElement; }; const attachNoteListeners = function attachNoteListeners(containerElement, editable = false) { const SELECT_PX_THRESHOLD = 4; const COLLAPSING_NOTE_DELAY = 750; const noteShadow = containerElement.shadowRoot; const noteElement = noteShadow.childNodes[1]; const headerElement = noteShadow.querySelector("header"); const mainElement = noteShadow.querySelector("textarea"); const noteId = containerElement.dataset.noteId; const resizeElement = noteShadow.querySelector(".note-resize"); const anchorIconElement = noteShadow.querySelector(".note-anchor"); const removeNoteElement = noteShadow.querySelector(".note-remove"); mainElement.readOnly = !editable; if (!editable) { anchorIconElement.style.setProperty("display", "none", "important"); } else { anchorIconElement.style.removeProperty("display"); } headerElement.ontouchstart = headerElement.onmousedown = event => { if (event.target == headerElement) { collapseNoteTimeout = setTimeout(() => noteElement.classList.toggle("note-collapsed"), COLLAPSING_NOTE_DELAY); event.preventDefault(); const position = getPosition(event); const clientX = position.clientX; const clientY = position.clientY; const boundingRect = noteElement.getBoundingClientRect(); const deltaX = clientX - boundingRect.left; const deltaY = clientY - boundingRect.top; maskPageElement.classList.add(PAGE_MASK_ACTIVE_CLASS); document.documentElement.style.setProperty("user-select", "none", "important"); anchorElement = getAnchorElement(containerElement); displayMaskNote(); selectNote(noteElement); moveNote(event, deltaX, deltaY); movingNoteMode = { event, deltaX, deltaY }; document.documentElement.ontouchmove = document.documentElement.onmousemove = event => { clearTimeout(collapseNoteTimeout); if (!movingNoteMode) { movingNoteMode = { deltaX, deltaY }; } movingNoteMode.event = event; moveNote(event, deltaX, deltaY); }; } }; resizeElement.ontouchstart = resizeElement.onmousedown = event => { event.preventDefault(); resizingNoteMode = true; selectNote(noteElement); maskPageElement.classList.add(PAGE_MASK_ACTIVE_CLASS); document.documentElement.style.setProperty("user-select", "none", "important"); document.documentElement.ontouchmove = document.documentElement.onmousemove = event => { event.preventDefault(); const { clientX, clientY } = getPosition(event); const boundingRectNote = noteElement.getBoundingClientRect(); noteElement.style.width = clientX - boundingRectNote.left + "px"; noteElement.style.height = clientY - boundingRectNote.top + "px"; }; }; anchorIconElement.ontouchend = anchorIconElement.onclick = event => { event.preventDefault(); noteElement.classList.toggle(NOTE_ANCHORED_CLASS); if (!noteElement.classList.contains(NOTE_ANCHORED_CLASS)) { deleteNoteRef(containerElement, noteId); addNoteRef(document.documentElement, noteId); } }; removeNoteElement.ontouchend = removeNoteElement.onclick = event => { event.preventDefault(); deleteNoteRef(containerElement, noteId); containerElement.remove(); }; noteElement.onmousedown = () => { selectNote(noteElement); }; function moveNote(event, deltaX, deltaY) { event.preventDefault(); const { clientX, clientY } = getPosition(event); noteElement.classList.add(NOTE_MOVING_CLASS); if (editable) { if (noteElement.classList.contains(NOTE_ANCHORED_CLASS)) { deleteNoteRef(containerElement, noteId); anchorElement = getTarget(clientX, clientY) || document.documentElement; addNoteRef(anchorElement, noteId); } else { anchorElement = document.documentElement; } } document.documentElement.insertBefore(containerElement, maskPageElement.getRootNode().host); noteElement.style.setProperty("left", (clientX - deltaX) + "px"); noteElement.style.setProperty("top", (clientY - deltaY) + "px"); noteElement.style.setProperty("position", "fixed"); displayMaskNote(); } function displayMaskNote() { if (anchorElement == document.documentElement || anchorElement == document.documentElement) { maskNoteElement.classList.remove(NOTE_MASK_MOVING_CLASS); } else { const boundingRectAnchor = anchorElement.getBoundingClientRect(); maskNoteElement.classList.add(NOTE_MASK_MOVING_CLASS); maskNoteElement.style.setProperty("top", boundingRectAnchor.y + "px"); maskNoteElement.style.setProperty("left", boundingRectAnchor.x + "px"); maskNoteElement.style.setProperty("width", boundingRectAnchor.width + "px"); maskNoteElement.style.setProperty("height", boundingRectAnchor.height + "px"); } } function selectNote(noteElement) { if (selectedNote) { selectedNote.classList.remove(NOTE_SELECTED_CLASS); } noteElement.classList.add(NOTE_SELECTED_CLASS); selectedNote = noteElement; } function getTarget(clientX, clientY) { const targets = Array.from(document.elementsFromPoint(clientX, clientY)).filter(element => element.tagName.toLowerCase() != NOTE_TAGNAME && !element.classList.contains(MASK_CLASS)); if (!targets.includes(document.documentElement)) { targets.push(document.documentElement); } let newTarget, target = targets[0], boundingRect = target.getBoundingClientRect(); newTarget = determineTargetElement("floor", target, clientX - boundingRect.left, getMatchedParents(target, "left")); if (newTarget == target) { newTarget = determineTargetElement("ceil", target, boundingRect.left + boundingRect.width - clientX, getMatchedParents(target, "right")); } if (newTarget == target) { newTarget = determineTargetElement("floor", target, clientY - boundingRect.top, getMatchedParents(target, "top")); } if (newTarget == target) { newTarget = determineTargetElement("ceil", target, boundingRect.top + boundingRect.height - clientY, getMatchedParents(target, "bottom")); } target = newTarget; while (boundingRect = target && target.getBoundingClientRect(), boundingRect && boundingRect.width <= SELECT_PX_THRESHOLD && boundingRect.height <= SELECT_PX_THRESHOLD) { target = target.parentElement; } return target; } function getMatchedParents(target, property) { let element = target, matchedParent, parents = []; do { const boundingRect = element.getBoundingClientRect(); if (element.parentElement && !element.parentElement.tagName.toLowerCase() != NOTE_TAGNAME && !element.classList.contains(MASK_CLASS)) { const parentBoundingRect = element.parentElement.getBoundingClientRect(); matchedParent = Math.abs(parentBoundingRect[property] - boundingRect[property]) <= SELECT_PX_THRESHOLD; if (matchedParent) { if (element.parentElement.clientWidth > SELECT_PX_THRESHOLD && element.parentElement.clientHeight > SELECT_PX_THRESHOLD && ((element.parentElement.clientWidth - element.clientWidth > SELECT_PX_THRESHOLD) || (element.parentElement.clientHeight - element.clientHeight > SELECT_PX_THRESHOLD))) { parents.push(element.parentElement); } element = element.parentElement; } } else { matchedParent = false; } } while (matchedParent && element); return parents; } function determineTargetElement(roundingMethod, target, widthDistance, parents) { if (Math[roundingMethod](widthDistance / SELECT_PX_THRESHOLD) <= parents.length) { target = parents[parents.length - Math[roundingMethod](widthDistance / SELECT_PX_THRESHOLD) - 1]; } return target; } }; const anchorNote = function anchorNote(event, noteElement, deltaX, deltaY) { event.preventDefault(); const { clientX, clientY } = getPosition(event); document.documentElement.style.removeProperty("user-select"); noteElement.classList.remove(NOTE_MOVING_CLASS); maskNoteElement.classList.remove(NOTE_MASK_MOVING_CLASS); maskPageElement.classList.remove(PAGE_MASK_ACTIVE_CLASS); const headerElement = noteElement.querySelector("header"); headerElement.ontouchmove = document.documentElement.onmousemove = null; let currentElement = anchorElement; let positionedElement; while (currentElement.parentElement && !positionedElement) { if (!FORBIDDEN_TAG_NAMES.includes(currentElement.tagName.toLowerCase())) { const currentElementStyle = getComputedStyle(currentElement); if (currentElementStyle.position != "static") { positionedElement = currentElement; } } currentElement = currentElement.parentElement; } if (!positionedElement) { positionedElement = document.documentElement; } const containerElement = noteElement.getRootNode().host; if (positionedElement == document.documentElement) { const firstMaskElement = document.querySelector("." + MASK_CLASS); document.documentElement.insertBefore(containerElement, firstMaskElement); } else { positionedElement.appendChild(containerElement); } const boundingRectPositionedElement = positionedElement.getBoundingClientRect(); const stylePositionedElement = window.getComputedStyle(positionedElement); const borderX = parseInt(stylePositionedElement.getPropertyValue("border-left-width")); const borderY = parseInt(stylePositionedElement.getPropertyValue("border-top-width")); noteElement.style.setProperty("position", "absolute"); noteElement.style.setProperty("left", (clientX - boundingRectPositionedElement.x - deltaX - borderX) + "px"); noteElement.style.setProperty("top", (clientY - boundingRectPositionedElement.y - deltaY - borderY) + "px"); }; const getPosition = function getPosition(event) { if (event.touches && event.touches.length) { const touch = event.touches[0]; return touch; } else { return event; } }; const onMouseUp = function onMouseUp(event) { if (highlightSelectionMode) { highlightSelection(); } if (removeHighlightMode) { let element = event.target, done; while (element && !done) { if (element.classList.contains(HIGHLIGHT_CLASS)) { document.querySelectorAll("." + HIGHLIGHT_CLASS + "[data-singlefile-highlight-id=" + JSON.stringify(element.dataset.singlefileHighlightId) + "]").forEach(highlightedElement => { resetHighlightedElement(highlightedElement); }); done = true; } element = element.parentElement; } } if (resizingNoteMode) { resizingNoteMode = false; document.documentElement.style.removeProperty("user-select"); maskPageElement.classList.remove(PAGE_MASK_ACTIVE_CLASS); document.documentElement.ontouchmove = document.documentElement.onmousemove = null; } if (movingNoteMode) { anchorNote(movingNoteMode.event || event, selectedNote, movingNoteMode.deltaX, movingNoteMode.deltaY); movingNoteMode = null; document.documentElement.ontouchmove = document.documentElement.onmousemove = null; } if (collapseNoteTimeout) { clearTimeout(collapseNoteTimeout); collapseNoteTimeout = null; } if (cuttingMode) { let element = event.target; if (document.documentElement != element && element.tagName.toLowerCase() != NOTE_TAGNAME) { element.classList.add(REMOVED_CONTENT_CLASS); removedElements.push(element); } } }; const maskNoteElement = getMaskElement("note-mask"); const maskPageElement = getMaskElement("page-mask", "single-file-page-mask"); let selectedNote, highlightSelectionMode, removeHighlightMode, resizingNoteMode, movingNoteMode, collapseNoteTimeout, cuttingMode; window.onresize = reflowNotes; document.documentElement.onmouseup = document.documentElement.ontouchend = onMouseUp; window.addEventListener("DOMContentLoaded", () => { processNode(document); reflowNotes(); document.querySelectorAll("single-file-note").forEach(noteElement => attachNoteListeners(noteElement)); }); })()</script><style>.single-file-highlight-yellow { background-color: #ffff7c !important; color: black !important; } span.single-file-highlight-yellow { display: inline !important; } .single-file-highlight-pink { background-color: #ffbbb6 !important; color: black !important; } span.single-file-highlight-pink { display: inline !important; } .single-file-highlight-blue { background-color: #95d0ff !important; color: black !important; } span.single-file-highlight-blue { display: inline !important; } .single-file-highlight-green { background-color: #93ef8d !important; color: black !important; } span.single-file-highlight-green { display: inline !important; } .single-file-highlight-hidden { background-color: inherit !important; color: inherit !important; } .single-file-mask { all: initial; display: contents !important; } .single-file-mask.single-file-page-mask { opacity: .99 !important; } single-file-note { all: initial !important; display: contents !important; } .single-file-hover { outline: 1px solid red !important; } .single-file-removed { display: none !important; float: none !important; position: static !important; visibility: collapse !important; }</style>